{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#source https://github.com/pytorch/examples/blob/master/mnist/main.py\n",
    "\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.optim.lr_scheduler import StepLR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Argument():\n",
    "    def __init__(self, batch_size=64, test_batch_size=1000,epochs=14, lr=1.0,\n",
    "                gamma=0.7,no_cuda=False, log_interval=100,save_model=False):\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.test_batch_size = test_batch_size\n",
    "        self.epochs = epochs\n",
    "        self.lr = lr\n",
    "        self.gamma = gamma\n",
    "        self.no_cuda = no_cuda\n",
    "        self.log_interval = log_interval\n",
    "        self.save_model = save_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 32, 3, 1)\n",
    "        self.conv2 = nn.Conv2d(32, 64, 3, 1)\n",
    "        self.dropout1 = nn.Dropout2d(0.25)\n",
    "        self.dropout2 = nn.Dropout2d(0.5)\n",
    "        self.fc1 = nn.Linear(9216, 128)\n",
    "        self.fc2 = nn.Linear(128, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.conv2(x)\n",
    "        x = F.max_pool2d(x, 2)\n",
    "        x = self.dropout1(x)\n",
    "        x = torch.flatten(x, 1)\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.dropout2(x)\n",
    "        x = self.fc2(x)\n",
    "        output = F.log_softmax(x, dim=1)\n",
    "        return output\n",
    "\n",
    "\n",
    "def train(args, model, device, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.nll_loss(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if batch_idx % args.log_interval == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item()))\n",
    "\n",
    "\n",
    "def test(args, model, device, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "\n",
    "\n",
    "def main():\n",
    "    args = Argument()\n",
    "    use_cuda = not args.no_cuda and torch.cuda.is_available()\n",
    "\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data', train=True, download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "        batch_size=args.batch_size, shuffle=True, **kwargs)\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "        batch_size=args.test_batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "    model = Net().to(device)\n",
    "    optimizer = optim.Adadelta(model.parameters(), lr=args.lr)\n",
    "\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)\n",
    "    for epoch in range(1, args.epochs + 1):\n",
    "        train(args, model, device, train_loader, optimizer, epoch)\n",
    "        test(args, model, device, test_loader)\n",
    "        scheduler.step()\n",
    "\n",
    "    if args.save_model:\n",
    "        torch.save(model.state_dict(), \"mnist_cnn.pt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 2.299212\n",
      "Train Epoch: 1 [6400/60000 (11%)]\tLoss: 0.277904\n",
      "Train Epoch: 1 [12800/60000 (21%)]\tLoss: 0.316506\n",
      "Train Epoch: 1 [19200/60000 (32%)]\tLoss: 0.176994\n",
      "Train Epoch: 1 [25600/60000 (43%)]\tLoss: 0.245506\n",
      "Train Epoch: 1 [32000/60000 (53%)]\tLoss: 0.058678\n",
      "Train Epoch: 1 [38400/60000 (64%)]\tLoss: 0.096332\n",
      "Train Epoch: 1 [44800/60000 (75%)]\tLoss: 0.166872\n",
      "Train Epoch: 1 [51200/60000 (85%)]\tLoss: 0.107573\n",
      "Train Epoch: 1 [57600/60000 (96%)]\tLoss: 0.174944\n",
      "\n",
      "Test set: Average loss: 0.0507, Accuracy: 9838/10000 (98%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.056642\n",
      "Train Epoch: 2 [6400/60000 (11%)]\tLoss: 0.032684\n",
      "Train Epoch: 2 [12800/60000 (21%)]\tLoss: 0.093746\n",
      "Train Epoch: 2 [19200/60000 (32%)]\tLoss: 0.010138\n",
      "Train Epoch: 2 [25600/60000 (43%)]\tLoss: 0.015474\n",
      "Train Epoch: 2 [32000/60000 (53%)]\tLoss: 0.100619\n",
      "Train Epoch: 2 [38400/60000 (64%)]\tLoss: 0.176754\n",
      "Train Epoch: 2 [44800/60000 (75%)]\tLoss: 0.041985\n",
      "Train Epoch: 2 [51200/60000 (85%)]\tLoss: 0.010226\n",
      "Train Epoch: 2 [57600/60000 (96%)]\tLoss: 0.114727\n",
      "\n",
      "Test set: Average loss: 0.0351, Accuracy: 9890/10000 (99%)\n",
      "\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.022554\n",
      "Train Epoch: 3 [6400/60000 (11%)]\tLoss: 0.150958\n",
      "Train Epoch: 3 [12800/60000 (21%)]\tLoss: 0.067651\n",
      "Train Epoch: 3 [19200/60000 (32%)]\tLoss: 0.092046\n",
      "Train Epoch: 3 [25600/60000 (43%)]\tLoss: 0.007468\n",
      "Train Epoch: 3 [32000/60000 (53%)]\tLoss: 0.058203\n",
      "Train Epoch: 3 [38400/60000 (64%)]\tLoss: 0.045174\n",
      "Train Epoch: 3 [44800/60000 (75%)]\tLoss: 0.101053\n",
      "Train Epoch: 3 [51200/60000 (85%)]\tLoss: 0.013102\n",
      "Train Epoch: 3 [57600/60000 (96%)]\tLoss: 0.115534\n",
      "\n",
      "Test set: Average loss: 0.0358, Accuracy: 9892/10000 (99%)\n",
      "\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.111951\n",
      "Train Epoch: 4 [6400/60000 (11%)]\tLoss: 0.051511\n",
      "Train Epoch: 4 [12800/60000 (21%)]\tLoss: 0.044360\n",
      "Train Epoch: 4 [19200/60000 (32%)]\tLoss: 0.054840\n",
      "Train Epoch: 4 [25600/60000 (43%)]\tLoss: 0.190840\n",
      "Train Epoch: 4 [32000/60000 (53%)]\tLoss: 0.003949\n",
      "Train Epoch: 4 [38400/60000 (64%)]\tLoss: 0.004146\n",
      "Train Epoch: 4 [44800/60000 (75%)]\tLoss: 0.039988\n",
      "Train Epoch: 4 [51200/60000 (85%)]\tLoss: 0.010134\n",
      "Train Epoch: 4 [57600/60000 (96%)]\tLoss: 0.004410\n",
      "\n",
      "Test set: Average loss: 0.0282, Accuracy: 9904/10000 (99%)\n",
      "\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.018205\n",
      "Train Epoch: 5 [6400/60000 (11%)]\tLoss: 0.054779\n",
      "Train Epoch: 5 [12800/60000 (21%)]\tLoss: 0.023917\n",
      "Train Epoch: 5 [19200/60000 (32%)]\tLoss: 0.007901\n",
      "Train Epoch: 5 [25600/60000 (43%)]\tLoss: 0.003488\n",
      "Train Epoch: 5 [32000/60000 (53%)]\tLoss: 0.063973\n",
      "Train Epoch: 5 [38400/60000 (64%)]\tLoss: 0.044873\n",
      "Train Epoch: 5 [44800/60000 (75%)]\tLoss: 0.088178\n",
      "Train Epoch: 5 [51200/60000 (85%)]\tLoss: 0.027690\n",
      "Train Epoch: 5 [57600/60000 (96%)]\tLoss: 0.006154\n",
      "\n",
      "Test set: Average loss: 0.0270, Accuracy: 9913/10000 (99%)\n",
      "\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.009428\n",
      "Train Epoch: 6 [6400/60000 (11%)]\tLoss: 0.039329\n",
      "Train Epoch: 6 [12800/60000 (21%)]\tLoss: 0.031136\n",
      "Train Epoch: 6 [19200/60000 (32%)]\tLoss: 0.000349\n",
      "Train Epoch: 6 [25600/60000 (43%)]\tLoss: 0.069548\n",
      "Train Epoch: 6 [32000/60000 (53%)]\tLoss: 0.003297\n",
      "Train Epoch: 6 [38400/60000 (64%)]\tLoss: 0.031523\n",
      "Train Epoch: 6 [44800/60000 (75%)]\tLoss: 0.009940\n",
      "Train Epoch: 6 [51200/60000 (85%)]\tLoss: 0.000732\n",
      "Train Epoch: 6 [57600/60000 (96%)]\tLoss: 0.005429\n",
      "\n",
      "Test set: Average loss: 0.0266, Accuracy: 9917/10000 (99%)\n",
      "\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.019150\n",
      "Train Epoch: 7 [6400/60000 (11%)]\tLoss: 0.023747\n",
      "Train Epoch: 7 [12800/60000 (21%)]\tLoss: 0.007934\n",
      "Train Epoch: 7 [19200/60000 (32%)]\tLoss: 0.014483\n",
      "Train Epoch: 7 [25600/60000 (43%)]\tLoss: 0.134040\n",
      "Train Epoch: 7 [32000/60000 (53%)]\tLoss: 0.010318\n",
      "Train Epoch: 7 [38400/60000 (64%)]\tLoss: 0.082867\n",
      "Train Epoch: 7 [44800/60000 (75%)]\tLoss: 0.008643\n",
      "Train Epoch: 7 [51200/60000 (85%)]\tLoss: 0.005474\n",
      "Train Epoch: 7 [57600/60000 (96%)]\tLoss: 0.006047\n",
      "\n",
      "Test set: Average loss: 0.0291, Accuracy: 9916/10000 (99%)\n",
      "\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.001457\n",
      "Train Epoch: 8 [6400/60000 (11%)]\tLoss: 0.005405\n",
      "Train Epoch: 8 [12800/60000 (21%)]\tLoss: 0.022044\n",
      "Train Epoch: 8 [19200/60000 (32%)]\tLoss: 0.013682\n",
      "Train Epoch: 8 [25600/60000 (43%)]\tLoss: 0.016523\n",
      "Train Epoch: 8 [32000/60000 (53%)]\tLoss: 0.003053\n",
      "Train Epoch: 8 [38400/60000 (64%)]\tLoss: 0.003021\n",
      "Train Epoch: 8 [44800/60000 (75%)]\tLoss: 0.002295\n",
      "Train Epoch: 8 [51200/60000 (85%)]\tLoss: 0.012886\n",
      "Train Epoch: 8 [57600/60000 (96%)]\tLoss: 0.090689\n",
      "\n",
      "Test set: Average loss: 0.0276, Accuracy: 9919/10000 (99%)\n",
      "\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.000344\n",
      "Train Epoch: 9 [6400/60000 (11%)]\tLoss: 0.007011\n",
      "Train Epoch: 9 [12800/60000 (21%)]\tLoss: 0.009486\n",
      "Train Epoch: 9 [19200/60000 (32%)]\tLoss: 0.008655\n",
      "Train Epoch: 9 [25600/60000 (43%)]\tLoss: 0.026839\n",
      "Train Epoch: 9 [32000/60000 (53%)]\tLoss: 0.007623\n",
      "Train Epoch: 9 [38400/60000 (64%)]\tLoss: 0.069004\n",
      "Train Epoch: 9 [44800/60000 (75%)]\tLoss: 0.080632\n",
      "Train Epoch: 9 [51200/60000 (85%)]\tLoss: 0.079804\n",
      "Train Epoch: 9 [57600/60000 (96%)]\tLoss: 0.003890\n",
      "\n",
      "Test set: Average loss: 0.0287, Accuracy: 9919/10000 (99%)\n",
      "\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.110675\n",
      "Train Epoch: 10 [6400/60000 (11%)]\tLoss: 0.068220\n",
      "Train Epoch: 10 [12800/60000 (21%)]\tLoss: 0.012710\n",
      "Train Epoch: 10 [19200/60000 (32%)]\tLoss: 0.004834\n",
      "Train Epoch: 10 [25600/60000 (43%)]\tLoss: 0.002673\n",
      "Train Epoch: 10 [32000/60000 (53%)]\tLoss: 0.005236\n",
      "Train Epoch: 10 [38400/60000 (64%)]\tLoss: 0.003050\n",
      "Train Epoch: 10 [44800/60000 (75%)]\tLoss: 0.010780\n",
      "Train Epoch: 10 [51200/60000 (85%)]\tLoss: 0.009700\n",
      "Train Epoch: 10 [57600/60000 (96%)]\tLoss: 0.022269\n",
      "\n",
      "Test set: Average loss: 0.0274, Accuracy: 9921/10000 (99%)\n",
      "\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.004842\n",
      "Train Epoch: 11 [6400/60000 (11%)]\tLoss: 0.001133\n",
      "Train Epoch: 11 [12800/60000 (21%)]\tLoss: 0.000888\n",
      "Train Epoch: 11 [19200/60000 (32%)]\tLoss: 0.000685\n",
      "Train Epoch: 11 [25600/60000 (43%)]\tLoss: 0.020944\n",
      "Train Epoch: 11 [32000/60000 (53%)]\tLoss: 0.001950\n",
      "Train Epoch: 11 [38400/60000 (64%)]\tLoss: 0.016221\n",
      "Train Epoch: 11 [44800/60000 (75%)]\tLoss: 0.105541\n",
      "Train Epoch: 11 [51200/60000 (85%)]\tLoss: 0.025386\n",
      "Train Epoch: 11 [57600/60000 (96%)]\tLoss: 0.070496\n",
      "\n",
      "Test set: Average loss: 0.0272, Accuracy: 9913/10000 (99%)\n",
      "\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.000155\n",
      "Train Epoch: 12 [6400/60000 (11%)]\tLoss: 0.035015\n",
      "Train Epoch: 12 [12800/60000 (21%)]\tLoss: 0.004849\n",
      "Train Epoch: 12 [19200/60000 (32%)]\tLoss: 0.007426\n",
      "Train Epoch: 12 [25600/60000 (43%)]\tLoss: 0.029482\n",
      "Train Epoch: 12 [32000/60000 (53%)]\tLoss: 0.002404\n",
      "Train Epoch: 12 [38400/60000 (64%)]\tLoss: 0.048970\n",
      "Train Epoch: 12 [44800/60000 (75%)]\tLoss: 0.058630\n",
      "Train Epoch: 12 [51200/60000 (85%)]\tLoss: 0.050304\n",
      "Train Epoch: 12 [57600/60000 (96%)]\tLoss: 0.001356\n",
      "\n",
      "Test set: Average loss: 0.0273, Accuracy: 9914/10000 (99%)\n",
      "\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.006950\n",
      "Train Epoch: 13 [6400/60000 (11%)]\tLoss: 0.002821\n",
      "Train Epoch: 13 [12800/60000 (21%)]\tLoss: 0.010991\n",
      "Train Epoch: 13 [19200/60000 (32%)]\tLoss: 0.010907\n",
      "Train Epoch: 13 [25600/60000 (43%)]\tLoss: 0.026268\n",
      "Train Epoch: 13 [32000/60000 (53%)]\tLoss: 0.079001\n",
      "Train Epoch: 13 [38400/60000 (64%)]\tLoss: 0.008126\n",
      "Train Epoch: 13 [44800/60000 (75%)]\tLoss: 0.000068\n",
      "Train Epoch: 13 [51200/60000 (85%)]\tLoss: 0.018658\n",
      "Train Epoch: 13 [57600/60000 (96%)]\tLoss: 0.002971\n",
      "\n",
      "Test set: Average loss: 0.0272, Accuracy: 9915/10000 (99%)\n",
      "\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.012173\n",
      "Train Epoch: 14 [6400/60000 (11%)]\tLoss: 0.035852\n",
      "Train Epoch: 14 [12800/60000 (21%)]\tLoss: 0.021906\n",
      "Train Epoch: 14 [19200/60000 (32%)]\tLoss: 0.009246\n",
      "Train Epoch: 14 [25600/60000 (43%)]\tLoss: 0.000496\n",
      "Train Epoch: 14 [32000/60000 (53%)]\tLoss: 0.007297\n",
      "Train Epoch: 14 [38400/60000 (64%)]\tLoss: 0.001963\n",
      "Train Epoch: 14 [44800/60000 (75%)]\tLoss: 0.019530\n",
      "Train Epoch: 14 [51200/60000 (85%)]\tLoss: 0.015998\n",
      "Train Epoch: 14 [57600/60000 (96%)]\tLoss: 0.001623\n",
      "\n",
      "Test set: Average loss: 0.0272, Accuracy: 9917/10000 (99%)\n",
      "\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 2.316960\n",
      "Train Epoch: 1 [6400/60000 (11%)]\tLoss: 0.178206\n",
      "Train Epoch: 1 [12800/60000 (21%)]\tLoss: 0.358732\n",
      "Train Epoch: 1 [19200/60000 (32%)]\tLoss: 0.158362\n",
      "Train Epoch: 1 [25600/60000 (43%)]\tLoss: 0.062494\n",
      "Train Epoch: 1 [32000/60000 (53%)]\tLoss: 0.177100\n",
      "Train Epoch: 1 [38400/60000 (64%)]\tLoss: 0.135936\n",
      "Train Epoch: 1 [44800/60000 (75%)]\tLoss: 0.059926\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [51200/60000 (85%)]\tLoss: 0.086300\n",
      "Train Epoch: 1 [57600/60000 (96%)]\tLoss: 0.057260\n",
      "\n",
      "Test set: Average loss: 0.0466, Accuracy: 9855/10000 (99%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.022379\n",
      "Train Epoch: 2 [6400/60000 (11%)]\tLoss: 0.091989\n",
      "Train Epoch: 2 [12800/60000 (21%)]\tLoss: 0.074851\n",
      "Train Epoch: 2 [19200/60000 (32%)]\tLoss: 0.012726\n",
      "Train Epoch: 2 [25600/60000 (43%)]\tLoss: 0.219235\n",
      "Train Epoch: 2 [32000/60000 (53%)]\tLoss: 0.039863\n",
      "Train Epoch: 2 [38400/60000 (64%)]\tLoss: 0.045708\n",
      "Train Epoch: 2 [44800/60000 (75%)]\tLoss: 0.092280\n",
      "Train Epoch: 2 [51200/60000 (85%)]\tLoss: 0.051113\n",
      "Train Epoch: 2 [57600/60000 (96%)]\tLoss: 0.019846\n",
      "\n",
      "Test set: Average loss: 0.0392, Accuracy: 9869/10000 (99%)\n",
      "\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.050926\n",
      "Train Epoch: 3 [6400/60000 (11%)]\tLoss: 0.044986\n",
      "Train Epoch: 3 [12800/60000 (21%)]\tLoss: 0.001592\n",
      "Train Epoch: 3 [19200/60000 (32%)]\tLoss: 0.019539\n",
      "Train Epoch: 3 [25600/60000 (43%)]\tLoss: 0.037825\n",
      "Train Epoch: 3 [32000/60000 (53%)]\tLoss: 0.042326\n",
      "Train Epoch: 3 [38400/60000 (64%)]\tLoss: 0.072836\n",
      "Train Epoch: 3 [44800/60000 (75%)]\tLoss: 0.012850\n",
      "Train Epoch: 3 [51200/60000 (85%)]\tLoss: 0.003129\n",
      "Train Epoch: 3 [57600/60000 (96%)]\tLoss: 0.289526\n",
      "\n",
      "Test set: Average loss: 0.0322, Accuracy: 9896/10000 (99%)\n",
      "\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.019107\n",
      "Train Epoch: 4 [6400/60000 (11%)]\tLoss: 0.041347\n",
      "Train Epoch: 4 [12800/60000 (21%)]\tLoss: 0.014986\n",
      "Train Epoch: 4 [19200/60000 (32%)]\tLoss: 0.000632\n",
      "Train Epoch: 4 [25600/60000 (43%)]\tLoss: 0.006828\n",
      "Train Epoch: 4 [32000/60000 (53%)]\tLoss: 0.192790\n",
      "Train Epoch: 4 [38400/60000 (64%)]\tLoss: 0.005678\n",
      "Train Epoch: 4 [44800/60000 (75%)]\tLoss: 0.084984\n",
      "Train Epoch: 4 [51200/60000 (85%)]\tLoss: 0.003457\n",
      "Train Epoch: 4 [57600/60000 (96%)]\tLoss: 0.003411\n",
      "\n",
      "Test set: Average loss: 0.0341, Accuracy: 9895/10000 (99%)\n",
      "\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.042601\n",
      "Train Epoch: 5 [6400/60000 (11%)]\tLoss: 0.009245\n",
      "Train Epoch: 5 [12800/60000 (21%)]\tLoss: 0.004236\n",
      "Train Epoch: 5 [19200/60000 (32%)]\tLoss: 0.012766\n",
      "Train Epoch: 5 [25600/60000 (43%)]\tLoss: 0.005438\n",
      "Train Epoch: 5 [32000/60000 (53%)]\tLoss: 0.006044\n",
      "Train Epoch: 5 [38400/60000 (64%)]\tLoss: 0.001837\n",
      "Train Epoch: 5 [44800/60000 (75%)]\tLoss: 0.079976\n",
      "Train Epoch: 5 [51200/60000 (85%)]\tLoss: 0.004242\n",
      "Train Epoch: 5 [57600/60000 (96%)]\tLoss: 0.000678\n",
      "\n",
      "Test set: Average loss: 0.0305, Accuracy: 9905/10000 (99%)\n",
      "\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.053780\n",
      "Train Epoch: 6 [6400/60000 (11%)]\tLoss: 0.023935\n",
      "Train Epoch: 6 [12800/60000 (21%)]\tLoss: 0.008180\n",
      "Train Epoch: 6 [19200/60000 (32%)]\tLoss: 0.008557\n",
      "Train Epoch: 6 [25600/60000 (43%)]\tLoss: 0.012294\n",
      "Train Epoch: 6 [32000/60000 (53%)]\tLoss: 0.036979\n",
      "Train Epoch: 6 [38400/60000 (64%)]\tLoss: 0.031068\n",
      "Train Epoch: 6 [44800/60000 (75%)]\tLoss: 0.006561\n",
      "Train Epoch: 6 [51200/60000 (85%)]\tLoss: 0.005691\n",
      "Train Epoch: 6 [57600/60000 (96%)]\tLoss: 0.007914\n",
      "\n",
      "Test set: Average loss: 0.0278, Accuracy: 9913/10000 (99%)\n",
      "\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.004523\n",
      "Train Epoch: 7 [6400/60000 (11%)]\tLoss: 0.025469\n",
      "Train Epoch: 7 [12800/60000 (21%)]\tLoss: 0.085680\n",
      "Train Epoch: 7 [19200/60000 (32%)]\tLoss: 0.003394\n",
      "Train Epoch: 7 [25600/60000 (43%)]\tLoss: 0.129341\n",
      "Train Epoch: 7 [32000/60000 (53%)]\tLoss: 0.004169\n",
      "Train Epoch: 7 [38400/60000 (64%)]\tLoss: 0.061636\n",
      "Train Epoch: 7 [44800/60000 (75%)]\tLoss: 0.006485\n",
      "Train Epoch: 7 [51200/60000 (85%)]\tLoss: 0.003548\n",
      "Train Epoch: 7 [57600/60000 (96%)]\tLoss: 0.009102\n",
      "\n",
      "Test set: Average loss: 0.0298, Accuracy: 9911/10000 (99%)\n",
      "\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.034508\n",
      "Train Epoch: 8 [6400/60000 (11%)]\tLoss: 0.013229\n",
      "Train Epoch: 8 [12800/60000 (21%)]\tLoss: 0.030964\n",
      "Train Epoch: 8 [19200/60000 (32%)]\tLoss: 0.003969\n",
      "Train Epoch: 8 [25600/60000 (43%)]\tLoss: 0.010571\n",
      "Train Epoch: 8 [32000/60000 (53%)]\tLoss: 0.021213\n",
      "Train Epoch: 8 [38400/60000 (64%)]\tLoss: 0.005418\n",
      "Train Epoch: 8 [44800/60000 (75%)]\tLoss: 0.087010\n",
      "Train Epoch: 8 [51200/60000 (85%)]\tLoss: 0.036207\n",
      "Train Epoch: 8 [57600/60000 (96%)]\tLoss: 0.005434\n",
      "\n",
      "Test set: Average loss: 0.0289, Accuracy: 9914/10000 (99%)\n",
      "\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.026266\n",
      "Train Epoch: 9 [6400/60000 (11%)]\tLoss: 0.005956\n",
      "Train Epoch: 9 [12800/60000 (21%)]\tLoss: 0.080699\n",
      "Train Epoch: 9 [19200/60000 (32%)]\tLoss: 0.000378\n",
      "Train Epoch: 9 [25600/60000 (43%)]\tLoss: 0.012772\n",
      "Train Epoch: 9 [32000/60000 (53%)]\tLoss: 0.006632\n",
      "Train Epoch: 9 [38400/60000 (64%)]\tLoss: 0.000066\n",
      "Train Epoch: 9 [44800/60000 (75%)]\tLoss: 0.002987\n",
      "Train Epoch: 9 [51200/60000 (85%)]\tLoss: 0.028229\n",
      "Train Epoch: 9 [57600/60000 (96%)]\tLoss: 0.095376\n",
      "\n",
      "Test set: Average loss: 0.0286, Accuracy: 9910/10000 (99%)\n",
      "\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.001151\n",
      "Train Epoch: 10 [6400/60000 (11%)]\tLoss: 0.018748\n",
      "Train Epoch: 10 [12800/60000 (21%)]\tLoss: 0.002121\n",
      "Train Epoch: 10 [19200/60000 (32%)]\tLoss: 0.064697\n",
      "Train Epoch: 10 [25600/60000 (43%)]\tLoss: 0.020952\n",
      "Train Epoch: 10 [32000/60000 (53%)]\tLoss: 0.006639\n",
      "Train Epoch: 10 [38400/60000 (64%)]\tLoss: 0.003118\n",
      "Train Epoch: 10 [44800/60000 (75%)]\tLoss: 0.001964\n",
      "Train Epoch: 10 [51200/60000 (85%)]\tLoss: 0.004107\n",
      "Train Epoch: 10 [57600/60000 (96%)]\tLoss: 0.033770\n",
      "\n",
      "Test set: Average loss: 0.0291, Accuracy: 9914/10000 (99%)\n",
      "\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.008600\n",
      "Train Epoch: 11 [6400/60000 (11%)]\tLoss: 0.186364\n",
      "Train Epoch: 11 [12800/60000 (21%)]\tLoss: 0.004458\n",
      "Train Epoch: 11 [19200/60000 (32%)]\tLoss: 0.052034\n",
      "Train Epoch: 11 [25600/60000 (43%)]\tLoss: 0.006427\n",
      "Train Epoch: 11 [32000/60000 (53%)]\tLoss: 0.036035\n",
      "Train Epoch: 11 [38400/60000 (64%)]\tLoss: 0.006845\n",
      "Train Epoch: 11 [44800/60000 (75%)]\tLoss: 0.007025\n",
      "Train Epoch: 11 [51200/60000 (85%)]\tLoss: 0.016632\n",
      "Train Epoch: 11 [57600/60000 (96%)]\tLoss: 0.001544\n",
      "\n",
      "Test set: Average loss: 0.0286, Accuracy: 9915/10000 (99%)\n",
      "\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.001358\n",
      "Train Epoch: 12 [6400/60000 (11%)]\tLoss: 0.020843\n",
      "Train Epoch: 12 [12800/60000 (21%)]\tLoss: 0.011861\n",
      "Train Epoch: 12 [19200/60000 (32%)]\tLoss: 0.028513\n",
      "Train Epoch: 12 [25600/60000 (43%)]\tLoss: 0.013780\n",
      "Train Epoch: 12 [32000/60000 (53%)]\tLoss: 0.002863\n",
      "Train Epoch: 12 [38400/60000 (64%)]\tLoss: 0.003148\n",
      "Train Epoch: 12 [44800/60000 (75%)]\tLoss: 0.071704\n",
      "Train Epoch: 12 [51200/60000 (85%)]\tLoss: 0.003770\n",
      "Train Epoch: 12 [57600/60000 (96%)]\tLoss: 0.075554\n",
      "\n",
      "Test set: Average loss: 0.0286, Accuracy: 9917/10000 (99%)\n",
      "\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.018418\n",
      "Train Epoch: 13 [6400/60000 (11%)]\tLoss: 0.001446\n",
      "Train Epoch: 13 [12800/60000 (21%)]\tLoss: 0.001125\n",
      "Train Epoch: 13 [19200/60000 (32%)]\tLoss: 0.080251\n",
      "Train Epoch: 13 [25600/60000 (43%)]\tLoss: 0.001465\n",
      "Train Epoch: 13 [32000/60000 (53%)]\tLoss: 0.020382\n",
      "Train Epoch: 13 [38400/60000 (64%)]\tLoss: 0.007662\n",
      "Train Epoch: 13 [44800/60000 (75%)]\tLoss: 0.048017\n",
      "Train Epoch: 13 [51200/60000 (85%)]\tLoss: 0.002799\n",
      "Train Epoch: 13 [57600/60000 (96%)]\tLoss: 0.014690\n",
      "\n",
      "Test set: Average loss: 0.0288, Accuracy: 9921/10000 (99%)\n",
      "\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.003275\n",
      "Train Epoch: 14 [6400/60000 (11%)]\tLoss: 0.000067\n",
      "Train Epoch: 14 [12800/60000 (21%)]\tLoss: 0.029446\n",
      "Train Epoch: 14 [19200/60000 (32%)]\tLoss: 0.011922\n",
      "Train Epoch: 14 [25600/60000 (43%)]\tLoss: 0.017904\n",
      "Train Epoch: 14 [32000/60000 (53%)]\tLoss: 0.015187\n",
      "Train Epoch: 14 [38400/60000 (64%)]\tLoss: 0.000866\n",
      "Train Epoch: 14 [44800/60000 (75%)]\tLoss: 0.002273\n",
      "Train Epoch: 14 [51200/60000 (85%)]\tLoss: 0.028717\n",
      "Train Epoch: 14 [57600/60000 (96%)]\tLoss: 0.004971\n",
      "\n",
      "Test set: Average loss: 0.0287, Accuracy: 9920/10000 (99%)\n",
      "\n",
      "2min 52s ± 38.1 ms per loop (mean ± std. dev. of 2 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n 1 -r 2\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
